{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting Whether a Planet Has a Shorter Year than Earth\n",
    "Using the Open Exoplanet Catalogue database: https://github.com/OpenExoplanetCatalogue/open_exoplanet_catalogue/\n",
    "\n",
    "## Data License\n",
    "Copyright (C) 2012 Hanno Rein\n",
    "\n",
    "Permission is hereby granted, free of charge, to any person obtaining a copy of this database and associated scripts (the \"Database\"), to deal in the Database without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Database, and to permit persons to whom the Database is furnished to do so, subject to the following conditions:\n",
    "\n",
    "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Database. A reference to the Database shall be included in all scientific publications that make use of the Database.\n",
    "\n",
    "THE DATABASE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE DATABASE OR THE USE OR OTHER DEALINGS IN THE DATABASE.\n",
    "\n",
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>period</th>\n",
       "      <th>name</th>\n",
       "      <th>eccentricity</th>\n",
       "      <th>description</th>\n",
       "      <th>discoverymethod</th>\n",
       "      <th>periastrontime</th>\n",
       "      <th>lastupdate</th>\n",
       "      <th>semimajoraxis</th>\n",
       "      <th>mass</th>\n",
       "      <th>periastron</th>\n",
       "      <th>list</th>\n",
       "      <th>discoveryyear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>326.03</td>\n",
       "      <td>11 Com b</td>\n",
       "      <td>0.231</td>\n",
       "      <td>11 Com b is a brown dwarf-mass companion to th...</td>\n",
       "      <td>RV</td>\n",
       "      <td>2452899.60</td>\n",
       "      <td>15/09/20</td>\n",
       "      <td>1.290</td>\n",
       "      <td>19.400</td>\n",
       "      <td>94.800</td>\n",
       "      <td>Confirmed planets</td>\n",
       "      <td>2008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>516.22</td>\n",
       "      <td>11 UMi b</td>\n",
       "      <td>0.080</td>\n",
       "      <td>11 Ursae Minoris is a star located in the cons...</td>\n",
       "      <td>RV</td>\n",
       "      <td>2452861.04</td>\n",
       "      <td>15/09/20</td>\n",
       "      <td>1.540</td>\n",
       "      <td>11.200</td>\n",
       "      <td>117.630</td>\n",
       "      <td>Confirmed planets</td>\n",
       "      <td>2009.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>185.84</td>\n",
       "      <td>14 And b</td>\n",
       "      <td>0.000</td>\n",
       "      <td>14 Andromedae is an evolved star in the conste...</td>\n",
       "      <td>RV</td>\n",
       "      <td>2452861.40</td>\n",
       "      <td>15/09/20</td>\n",
       "      <td>0.830</td>\n",
       "      <td>4.800</td>\n",
       "      <td>0.000</td>\n",
       "      <td>Confirmed planets</td>\n",
       "      <td>2008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1766.00</td>\n",
       "      <td>14 Her b</td>\n",
       "      <td>0.359</td>\n",
       "      <td>The star 14 Herculis is only 59 light years aw...</td>\n",
       "      <td>RV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15/09/21</td>\n",
       "      <td>2.864</td>\n",
       "      <td>4.975</td>\n",
       "      <td>22.230</td>\n",
       "      <td>Confirmed planets</td>\n",
       "      <td>2002.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9886.00</td>\n",
       "      <td>14 Her c</td>\n",
       "      <td>0.184</td>\n",
       "      <td>14 Her c is the second companion in the system...</td>\n",
       "      <td>RV</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15/09/21</td>\n",
       "      <td>9.037</td>\n",
       "      <td>7.679</td>\n",
       "      <td>189.076</td>\n",
       "      <td>Controversial</td>\n",
       "      <td>2006.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    period      name  eccentricity  \\\n",
       "0   326.03  11 Com b         0.231   \n",
       "1   516.22  11 UMi b         0.080   \n",
       "2   185.84  14 And b         0.000   \n",
       "3  1766.00  14 Her b         0.359   \n",
       "4  9886.00  14 Her c         0.184   \n",
       "\n",
       "                                         description discoverymethod  \\\n",
       "0  11 Com b is a brown dwarf-mass companion to th...              RV   \n",
       "1  11 Ursae Minoris is a star located in the cons...              RV   \n",
       "2  14 Andromedae is an evolved star in the conste...              RV   \n",
       "3  The star 14 Herculis is only 59 light years aw...              RV   \n",
       "4  14 Her c is the second companion in the system...              RV   \n",
       "\n",
       "   periastrontime lastupdate  semimajoraxis    mass  periastron  \\\n",
       "0      2452899.60   15/09/20          1.290  19.400      94.800   \n",
       "1      2452861.04   15/09/20          1.540  11.200     117.630   \n",
       "2      2452861.40   15/09/20          0.830   4.800       0.000   \n",
       "3             NaN   15/09/21          2.864   4.975      22.230   \n",
       "4             NaN   15/09/21          9.037   7.679     189.076   \n",
       "\n",
       "                list  discoveryyear  \n",
       "0  Confirmed planets         2008.0  \n",
       "1  Confirmed planets         2009.0  \n",
       "2  Confirmed planets         2008.0  \n",
       "3  Confirmed planets         2002.0  \n",
       "4      Controversial         2006.0  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "planets = pd.read_csv('../../ch_09/data/planets.csv')\n",
    "planets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the `shorter_year_than_earth` column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "planets['shorter_year_than_earth'] = planets.period < planets.query('name == \"Earth\"').period.iat[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     3258\n",
       "False     556\n",
       "Name: shorter_year_than_earth, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "planets.shorter_year_than_earth.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9696969696969697"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "data = planets[['shorter_year_than_earth', 'semimajoraxis', 'mass', 'eccentricity']].dropna()\n",
    "y = data.pop('shorter_year_than_earth')\n",
    "X = data\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.25, random_state=0, stratify=y\n",
    ")\n",
    "\n",
    "lm = LogisticRegression(solver='lbfgs', random_state=0).fit(X_train, y_train)\n",
    "lm.score(X_test, y_test) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = lm.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False       0.95      0.97      0.96        98\n",
      "        True       0.98      0.97      0.98       166\n",
      "\n",
      "   micro avg       0.97      0.97      0.97       264\n",
      "   macro avg       0.97      0.97      0.97       264\n",
      "weighted avg       0.97      0.97      0.97       264\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x111ce2d0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4FFXWwOHfSQgEZF9cMGxCECEkgAERFBUEkSUyI6MgLmjYxW3UAbcPcFwQN0RBQMLgguiIKAFBUAQRBmSRHVR2CIJsAdkCJDnfH9VpYwjpDkmn053zPk8/dlXfqjrVkTp97626V1QVY4wxBiDE3wEYY4wpPCwpGGOMcbOkYIwxxs2SgjHGGDdLCsYYY9wsKRhjjHGzpGCMMcbNkoIJOiKyQ0ROichxEdknIpNEpHSWMi1E5DsROSYiR0VkhojUz1KmrIiMFJFdrn1tcS1XLtgzMqbgWFIwwaqzqpYGGgGNgacyPhCRa4G5wHSgKlALWAMsFpErXGWKA/OABkB7oCzQAjgENPNV0CJSzFf7NsYblhRMUFPVfcAcnOSQYQTwgaq+parHVPWwqj4LLAWGusrcC1QH/qaqG1U1XVX3q+q/VXVWdscSkQYi8o2IHBaR30Xkadf6SSLyQqZyN4pIUqblHSIySETWAidE5FkRmZpl32+JyCjX+3IikiAie0Vkj4i8ICKhefyqjAEsKZggJyIRwK3AFtdyKZxf/J9lU/y/QFvX+5uBr1X1uJfHKQN8C3yNU/uog1PT8FZ3oCNQHvgQ6CAiZV37DgXuAD52lX0fSHUdozHQDuiVi2MZc16WFEyw+lJEjgG7gf3AENf6ijj/3+/NZpu9QEZ/QaXzlDmfTsA+VX1dVVNcNZAfc7H9KFXdraqnVHUn8BPQxfVZa+Ckqi4VkUtwktyjqnpCVfcDbwLdcnEsY87LkoIJVl1UtQxwI1CPPy/2yUA6cFk221wGHHS9P3SeMudTDdh6QZE6dmdZ/hin9gBwF3/WEmoAYcBeETkiIkeAccDFeTi2MW6WFExQU9XvgUnAa67lE8AS4B/ZFL+DP5t8vgVuEZGLvDzUbqD2eT47AZTKtHxpdqFmWf4MuNHV/PU3/kwKu4HTQGVVLe96lVXVBl7GaUyOLCmYomAk0FZEMjqbBwP3icjDIlJGRCq4OoKvBYa5ynyIcwH+XETqiUiIiFQSkadFpEM2x5gJXCoij4pICdd+r3F9thqnj6CiiFwKPOopYFU9ACwA/gNsV9VNrvV7ce6cet11y2yIiNQWkRsu4Hsx5hyWFEzQc11gPwCecy0vAm4B/o7Tb7ATp8P2OlXd7CpzGqez+WfgG+APYBlOM9Q5fQWqegynk7ozsA/YDNzk+vhDnFted+Bc0D/1MvSPXTF8nGX9vUBxYCNOc9hUctfUZcx5iU2yY4wxJoPVFIwxxrhZUjDGGONmScEYY4ybJQVjjDFuATf4VuXKlbVmzZr+DsMYYwLKypUrD6pqFU/lAi4p1KxZkxUrVvg7DGOMCSgistObctZ8ZIwxxs2SgjHGGDdLCsYYY9wsKRhjjHGzpGCMMcbNZ0lBRCaKyH4RWX+ez0VERrkmQ18rIk18FYsxxhjv+LKmMAlnwvPzuRWIdL36AO/6MBZjjDFe8NlzCqq6UERq5lDkNpzJ0xVYKiLlReQy13jxRVPHjjAr2znhjTHG4eORrf3Zp3A5f52CMMm17hwi0kdEVojIigMHDhRIcHnWsSOI5O5lCcEY42f+TAqSzbpsU6CqjlfVWFWNrVLF41PahcOFXuA7dHB+CdjLXvaylyr/enIuoSHDGPSvuaScOpu/16ls+HOYiyScyc4zRAC/+SmWC+epyUe14GIxxgS8I0dS2LYtmSZNnMn0hg27kW7dotzLvubPmkIicK/rLqTmwNGA7E/IKSF0yG4qX2OMyd706T9Tv/5o4uKmcPRoCgAlS4YVWEIAH9YURGQKcCNQWUSSgCFAGICqjgVmAR2ALcBJ4H5fxXLBctPxazUCY8wF2r//BA8/PJtPP90AQPPmERw5kkK5cuEFHosv7z7q7uFzBR701fEv2IXcAWQ1AmPMBVBVJk9exyOPfM3hw6coVSqMl15qzcCBzQgN9U9DTsANne0TOSWCDh3gq68KNh5jTJHQv/9XjBu3EoCbb76C8eM7UatWBb/GZMNcwLkJIfMdQJYQjDE+0qVLPcqXDychIY65c+/2e0IAqyn8lfULGGN8aPPmQ8ybt51+/WIBaN++Djt2POKXvoPzKZo1hawPlhljjA+lpqYzYsRioqPHMmDAVyxdmuT+rDAlBCiqNYXs+g+ss9gY4wNr1uwjPj6RlSudO+7vvTeGyMiKfo7q/IpmUshgzUXGGB85fTqVF15YyPDhi0lNTad69XKMG9eJ9u3r+Du0HBXtpGCMMT7y1FPzePPNpQA8+GBTXn65DWXKlPBzVJ5ZUjDGGB/4179asmRJEiNG3Mz119fwdzheK5odzcYYk8+++WYrt9/+X1JT0wG49NLS/O9/DwRUQgBLCsYYkyfJyaeIj59Ou3YfMW3aJv7zn1XuzyQA72605iNjjLlAX3yxiQEDZrFv33FKlAhlyJAb6Nmzkb/DyhNLCsYYk0v79h3noYdmM3XqRgBatKhGQkIc9epV9nNkeWdJwRhjcmn69J+ZOnUjF10UxvDhNzNgQFNCQgKvqSg7lhSMMcYLKSmphIc7l8zeva9m27Zk+vdvSs2a5f0cWf6yjmZjjMlBerryzjvLqFXrLXbuPAJASIjwyittgy4hgCUFY4w5r19+OUirVv/hoYdms2/fcaZMWe/vkHzOmo+MMSaLs2fTeO21/zFs2PecPp3GJZdcxJgxHfn736/yd2g+Z0nBGGMyWb9+P/fe+wWrVu0D4P77G/H66+2oUKGknyMrGJYUjDEmk/R0Zd26/dSoUY7x4zvTrl1tf4dUoCwpGGOKvA0b9lO/fhVEhOjoS5g+vRutWtWgdOni/g6twFlHszGmyDp27DQDB84iKupdPv98k3t9hw6RRTIhgNUUjDFF1Jw5W+jTZya7dh2lWLEQduw44u+QCgVLCsaYIuXw4VM89tgcPvhgDQBNmlxGQkIcjRpd6ufICgdLCsaYImP16n20b/8Rv/9+ghIlQhk27EYef7wFxYpZS3oGSwrGmCKjbt1KlC5dnLp1KzFhQhx161byd0iFjiUFY0zQUlU+/ngdnTtfSdmyJShVKowFC3pStWqZoBnALr9ZnckYE5R27DjCLbd8xN13f8Hgwd+610dElLWEkAOrKRhjgkpaWjpjxiznqafmceLEWSpWLEmLFtX8HVbAsKRgjAkamzYdID4+kSVLkgC4444GvP32rVx88UV+jixwWFIwxgSF7duTadRoHGfOpHHZZaUZM6YjXbrU83dYAceSgjEmKNSqVYF//KM+4eHFeO21dpQvH+7vkAKSTzuaRaS9iPwiIltEZHA2n1cXkfkiskpE1opIB1/GY4wJHqdOneWpp75l2bI97nXvv9+FCRPiLCHkgc+SgoiEAqOBW4H6QHcRqZ+l2LPAf1W1MdANGOOreIwxweOHH3bSqNE4hg9fTJ8+M0hPVwBCQ+2Gyrzy5TfYDNiiqttU9QzwCXBbljIKlHW9Lwf85sN4jDEB7o8/TvPgg1/RqtUkfv31EPXrV2Hs2E52i2k+8mWfwuXA7kzLScA1WcoMBeaKyEPARcDN2e1IRPoAfQCqV6+e74EaYwq/WbM206/fTHbv/oNixUJ4+unrePrp6ylRwrpG85PHb1NEigMdgOuBqsApYD0wS1V/zmnTbNZpluXuwCRVfV1ErgU+FJEoVU3/y0aq44HxALGxsVn3YYwJckePptCjxzSOHEkhNrYqCQlxREdf4u+wglKOSUFEngVuBxYCK4FvgHCgLvCmiAjwhKpmN5t1EpD5iZEIzm0eigfaA6jqEhEJByoD+3N/KsaYYKKqqEJIiFCuXDijRrXn999P8OijzW0AOx/yVFNYp6ovnOezESJyGX+98Ge2HIgUkVrAHpyO5LuylNkFtAEmichVOAnngFeRG2OC1m+/HWPAgK+4/vrqPP54CwDuuSfGz1EVDTmmW1Wdfr7PRCRCVfeq6rLzbJsKDATmAJtw7jLaICLPi0icq9jjQG8RWQNMAXqqqjUPGVNEqSoJCT9Rv/5opk//hVdf/R+nTp31d1hFijd9Ck1xOo0XqepBEWkADAJa4zQJnZeqzgJmZVn3f5nebwRaXkDcxpggs21bMr17z+C777YD0LFjJGPHdqJkyTA/R1a05FhTEJGXgclAD+BrEXkGmA+swelXMMaYPElLS+fNN5cQFTWG777bTuXKpfj4478zY0Z3IiLKet6ByVeeagq3ATGqekpEKuJ0FMeo6i++D80YU1RMnbqJU6dS6d49irfeak+VKjaAnb94SgopqnoKQFUPi8jPlhCMMXl15kwax46dplKlUoSGhpCQEMfmzYfo3PlKf4dW5HlKCleIyDTXewFqZlpGVf/us8iMMUFp+fI9xMcnEhFRlq++ugsRoV69ytSrV9nfoRk8J4Xbsyy/46tAjDHB7eTJswwZMp833lhKerpy8uRZ9u8/wSWXlPZ3aCaTHJOCqs4TkYZAbWCDqm4umLCMMcFkwYId9O49gy1bDhMSIjzxxLUMG3YTpUrZnUWFjacnmp/Geer4J6CpiDyvqhMLJDJjTMBTVR5+eDbvvLMcgIYNLyYhIY6mTS/3c2TmfDw1H/UAolX1hIhUwXnmwJKCMcYrIkLZsiUICwvh2WdbMXjwdRQvHurvsEwOPCWF06p6AkBVD4iIDThijMnRwYMn2br1MNdc4zzb+txzN9CjRzT161fxc2TGG7m9+6i23X1kjMmOqvLppxt46KHZFCsWwsaNA6hQoSTh4cUsIQQQu/vIGJNnSUl/MGDAV8yY8SsArVvX4uTJs1SoUNLPkZnc8pQU7lLV+AKJxBgTcNLTlQkTfuLJJ7/hjz9OU7ZsCV5/vR3x8Y1xRtY3gcZTUmhcIFEYYwJSfHwikyatBiAu7krGjOnA5ZfbeEWBzFNSKOV6TiHblK+qa/M/JGNMoLj77obMmrWZUaPac8cdDax2EAQ8JYXLgdGcf2rNVvkekTGm0Fq/fj/z5m3jkUeaA9CmzRVs2/YwF11U3M+RmfziKSlsUVW78BtTxJ0+ncrLLy/ipZd+4OzZdGJjq9KyZXUASwhBxuMkO8aYou3HH5OIj09kwwZnptz+/WNp2PASP0dlfMVTUni6QKIwxhQ6J06c4bnn5jNy5FJUITKyIhMmxNGqVQ1/h2Z8yFNS6OvqOPrGNeeym4jUAO4Dkmw8JGOCzzPPfMdbb/1ISIjw5JPXMnTojTY1ZhHgKSk8CDwOjBaR34EDQDhQC9gNjFbVz30bojHGH5555nrWrdvPK6/cTGxsVX+HYwqIqKp3BUXqAJcBp4BfVPWYLwM7n9jYWF2xYkXedpJx25yX525MUZCY+Atjx65g+vRuhIXZoHXBRkRWqmqsp3JedzSr6hZgS56iMsYUOvv3n+Dhh2fz6acbAHj//TX06tXEz1EZf7G7j4wpolSVyZPX8cgjX3P48ClKlQrj5ZfbcP/9jfwdmvEjSwrGFEG7dh2lX7+ZzJ7tVP5vvvkKxo/vRK1aFfwcmfE3r5OCiBQHqruakYwxAWzu3K3Mnr2F8uXDeeONdvTs2ciGqDCAl0lBRDoCbwDFgVoi0ggYoqp/82Vwxpj8c+LEGffTx/Hxjdmz5w/69Lmayy4r4+fITGHi7UxqzwPXAEcAVHU1UMdXQRlj8k9qajojRiymRo2RbNuWDDjTZA4ZcqMlBHMOb5PCWVU9kmWd3c9pTCG3Zs0+rrlmAoMGfcuhQ6f48suf/R2SKeS87VPYJCJ3ACEiUgt4BFjqu7CMMXlx+nQqL7ywkOHDF5Oamk716uUYP74Tt9xiFXyTM2+TwkDg/4B0YBowB3jKV0EZYy7cqlV76dFjGps2HUQEBg5syksvtaFMmRL+Ds0EAG+bj25R1UGq2tj1Ggzc6mkjEWkvIr+IyBYRGXyeMneIyEYR2SAiH+cmeGPMuUqUKMbWrclceWUlFi68n7ff7mAJwXjN26TwbDbrnslpAxEJxZmg51agPtBdROpnKROJU+NoqaoNgEe9jMcYk8lPP+0lY8ia+vWrMHt2D1av7sd111X3c2Qm0OTYfCQitwDtgctF5I1MH5XFaUrKSTOcSXq2ufb1CXAbsDFTmd44g+olA6jq/tyFb0zRlpx8iieemMvEiauZMuV2unWLAqB161p+jswEKk99CvuB9UAKsCHT+mNAts1BmVyOM5JqhiSc21ozqwsgIouBUGCoqn6ddUci0gfoA1C9uv3yMQbgiy82MWDALPbtO06JEqEcOnTS3yGZIJBjUlDVVcAqEZmsqim53Pf55nXOevxI4EYgAvhBRKKy3v6qquOB8eCMkprLOIwJKvv2Heehh2YzdapT6W7ZshoTJsRRr15lP0dmgoG3dx9dLiIv4vQNhGesVNW6OWyTBFTLtBwB/JZNmaWqehbYLiK/4CSJ5V7GZUyRsnLlb7Rt+yHJySlcdFEYw4ffzIABTQkJsSEqTP7wtqN5EvAfnF//twL/BT7xsM1yIFJEarnGTeoGJGYp8yVwE4CIVMZpTtrmZUzGFDn161ehSpWLuOWW2mzYMICBA5tZQjD5ytukUEpV5wCo6lZVfRbXxfx8XNN3DsR5pmET8F9V3SAiz4tInKvYHOCQiGwE5gNPquqhCzkRY4JReroyfvxKjhxxWm9Llgxj4cKezJ7dgxo1yvs5OhOMvG0+Oi3OEIpbRaQfsAe42NNGqjoLmJVl3f9leq/AP10vY0wmv/xykF69ZrBo0S6WL9/De+85v6UuuaS0nyMzwczbpPAYUBp4GHgRKAc84KugjCnKzp5N4/XXlzB06AJOn07j0ktLc+utkf4OyxQRXiUFVf3R9fYYcA+AiET4KihjiqpVq/YSH5/IqlX7ALj//ka8/no7KlQo6efITFHhMSmISFOcZw4WqepBEWkADAJa49xRZIzJB1u3HqZZswmkpqZTs2Z5xo/vRNu2tf0dliliPD3R/DJwO7AGeFZEvsAZIfUVoJ/vwzOm6KhduyL33BNNmTLFefHFNpQuXdzfIZkiyFNN4TYgRlVPiUhFnOcMYlT1F9+HZkxwO378DE8/PY/u3aO49lrnkZ6EhDibFtP4laekkKKqpwBU9bCI/GwJwZi8mzNnC336zGTXrqN8//1OVq/ui4hYQjB+5ykpXCEi01zvBaiZaRlV/bvPIjMmCB0+fIrHHpvDBx+sAeDqqy+z2oEpVDwlhduzLL/jq0CMCXZTp27kwQdnsX//CcLDizFs2I3885/XUqyYt8+QGuN7ngbEm1dQgRgTzI4cSaFPnxkkJ6fQqlUN3nuvM3XrVvJ3WMacw9uH14wxuaSqpKcroaEhlC8fzpgxHUlOPkXfvrE2XpEptCwpGOMDO3YcoU+fGbRuXYvBg68DcE+AY0xhlqvGTBGxiV6NyUFaWjqjRv1IVNQYvvlmG++8s4yUlFR/h2WM17xKCiLSTETWAZtdyzEi8rZPIzMmwGzadIBWrSbxyCNfc+LEWbp1i+Knn/oSHm4VchM4vP2/dRTQCWf+A1R1jYjkOHS2MUVFamo6r7yyiOefX8iZM2lUrVqGd9/tSFzclf4OzZhc8zYphKjqziz3Uqf5IB5jAk5IiDB37jbOnEmjd+8mjBjRlvLlwz1vaEwh5G1S2C0izQAVkVDgIeBX34VlTOF26tRZjh07w8UXX0RIiDBhQmd27/6D1q1r+Ts0Y/LE247m/jgT4VQHfgeau9YZU+QsXLiTmJix3H33NJx5oiAyspIlBBMUvK0ppKpqN59GYkwh98cfp3nqqW8ZM2YFAGFhoRw8eJIqVS7yc2TG5B9vk8JyEfkF+BSYpqrHfBiTMYXO7Nmb6dt3Jrt3/0GxYiE888z1PPXUdZQoYXcWmeDi7cxrtUWkBdANGCYiq4FPVPUTn0ZnjJ+pKr17zyAhYRUAsbFVmTgxjoYNL/FzZMb4htcPr6nq/1T1YaAJ8Acw2WdRGVNIiAgREWUJDy/Ga6+1ZcmSeEsIJqh5VVMQkdI4E+50A64CpgMtfBiXMX7z22/H2Lr1MNdfXwOAp5++nnvuiaZ27Yp+jswY3/O2QXQ9MAMYoao/+DAeY/xGVZk4cRWPPz6X4sVD2bTpQSpVKkXx4qGWEEyR4W1SuEJV030aiTF+tG1bMr17z+C777YD0KlTXc6etf/lTdGTY1IQkddV9XHgcxHRrJ/bzGsm0GUMYPfss/M5efIslSuXYtSo9nTrFmWzoZkiyVNN4VPXf23GNROU7r33Sz7+eB0Ad93VkJEjb7HnDkyR5mnmtWWut1ep6l8Sg4gMBGxmNhPQevduwsKFOxkzpgOdO9sAdsZ4e0vqA9msi8/PQIwpCMuX7+GVVxa5l2+8sSZbtjxkCcEYF099Cnfi3IZaS0SmZfqoDHDEl4EZk59OnjzLkCHzeeONpaSnKy1aVHPfcmpPJRvzJ0//GpYBh4AIYHSm9ceAVb4Kypj8tGDBDnr1SmTr1mRCQoQnnriWq6+u6u+wjCmUPPUpbAe2A98WTDjG5J+jR1P417++Yfz4nwBo2PBiEhLiaNr0cj9HZkzhlWOfgoh87/pvsogczvRKFpHDnnYuIu1F5BcR2SIig3Mo11VEVERic38KxmTvuefmM378T4SFhfD88zeyYkUfSwjGeOCp+Shjys3Kud2xazKe0UBbIAlnpNVEVd2YpVwZ4GHgx9wew5isVNX9fMH//d8NbN9+hOHD29CgwcV+jsyYwJBjTSHTU8zVgFBVTQOuBfoCnm7mbgZsUdVtqnoG+ARn/KSs/g2MAFJyE7gxmakqH3+8jtatP+DMGWem2MqVSzFjRndLCMbkgre3pH6JMxVnbeADnEHxPvawzeXA7kzLSa51biLSGKimqjNz2pGI9BGRFSKy4sCBA16GbIqKpKQ/iIv7hB49prFgwQ4mT17r75CMCVjeJoV0VT0L/B0YqaoPkeUCn43sxghwD5UhIiHAm8Djng6uquNVNVZVY6tUqeJlyCbYpacr48atoH790cyc+SvlypVgwoTO9OzZyN+hGROwvJ6OU0T+AdwDdHGtC/OwTRJOs1OGCOC3TMtlgChggasN+FIgUUTiVHWFl3GZImrLlsP07j2DBQt2AHDbbVcyZkxHqlYt49/AjAlw3iaFB4ABOENnbxORWsAUD9ssByJdZffgPAR3V8aHqnqUTB3YIrIAeMISgvHGDz/sZMGCHVx88UW8886tdO1a3wawMyYfeDsd53oReRioIyL1cDqQX/SwTaprfKQ5QCgwUVU3iMjzwApVTcxr8KZoOXIkhfLlwwHo2bMRBw6cJD6+MZUqlfJzZMYED1E9Z0TscwuJXA98iPOLX3Caeu5R1cW+De9csbGxumJFHisTGb8ovTh343+nT6fy0ks/MHLkj6xY0ZvIyEr+DsmYgCMiK1XV47Ng3jYfvQl0yHjGQESuwkkS9rCZ8amlS5OIj09k40bnrrM5c7ZaUjDGh7xNCsUzP3SmqptEpLiPYjKGEyfO8Nxz8xk5cimqEBlZkYSEOPcgdsYY3/A2KfwkIuNwagcAPbAB8YyP/PhjEnfdNY1t25IJDRWeeKIFQ4bcQMmSnm54M8bklbdJoR/OUBT/wulTWAi87augTNFWvnw4e/b8QUzMJSQkxNmIpsYUII9JQUQaArWBL1R1hO9DMkXRokW7aNmyGiLClVdW5rvv7qNp06qEhYX6OzRjihRPo6Q+jTPERQ/gGxHJbgY2Yy7Y/v0n6NZtKtdf/x8+/PDP4SlatKhmCcEYP/BUU+gBRKvqCRGpAswCJvo+LBPsVJXJk9fxyCNfc/jwKUqVCnMPZGeM8R9PSeG0qp4AUNUDrvGKjMmTXbuO0q/fTGbP3gJA27ZXMH58Z2rWLO/nyIwxnpLCFZnmZhagdua5mlX17z6LzASlH39M4uabP+T48TOULx/Om2/ewn33xdgQFcYUEp6Swu1Zlt/xVSCmaGjU6FKqVStLvXqVGT26A5ddZgPYGVOYeJqjeV5BBWKCU2pqOu+8s4x7742hYsWSlChRjMWLH6BChZL+Ds0Ykw1vn1MwJtfWrNnHAw8k8tNPe1m9eh+TJjmjrltCMKbwsqRg8l1KSiovvLCQV15ZTGpqOtWrl6N79yh/h2WM8UKukoKIlFDV074KxgS+//1vN/Hxifz880FEYODAprz0UhvKlCnh79CMMV7wKimISDMgASgHVBeRGKCXa1pOYwBnNrTrr/8P6enKlVdWIiEhjpYtq/s7LGNMLnhbUxgFdMJ5uhlVXSMiN/ksKhOQ6tSpSJ8+TahYsSTPPXcD4eHWOmlMoPH2X22Iqu7Mci+5PX5axCUnn+Lxx+dy//2N3ENajxnT0Z45MCaAeZsUdruakFREQoGHgF99F5Yp7KZN28SDD85i377jrFy5l9Wr+yIilhCMCXDeJoX+OE1I1YHfgW9d60wRs2/fcQYOnMXnn28C4LrrqjNhQmdLBsYECa+SgqruB7r5OBZTiKkqH3ywhscem0NycgqlSxfnlVdupl+/WEJCLCEYEyy8vfvoPeCcWe5VtU++R2QKpSNHUnj88bkkJ6fQvn0dxo7tSI0aNoCdMcHG2+ajbzO9Dwf+BuzO/3BMYZKerqSnK8WKhVChQknGjevEyZNnufvuaGsuMiZIedt89GnmZRH5EPjGJxGZQuHnnw/Sq1ci7dvX4dlnWwFw++31/RyVMcbXLnR+hFpAjfwMxBQOZ8+m8dJLPxATM5bFi3eTkLCKlJRUf4dljCkg3vYpJPNnn0IIcBgY7KugjH+sWrWXBx5IZPXqfQDExzfm1Vfb2kNoxhQhHv+1i9N4HAPsca1KV9VzOp1N4Dp7No0hQxYwYsRi0tKUmjXL8957nbn55iv8HZoxpoB5bD5yJYAvVDXN9bKEEGSKFQvhxx/3kJ6uPPLINaxb198SgjFFlLftAstEpImq/uTTaEyBOXbsNMeOnaFq1TKICBMmdGbfvuNce21fchh1AAAZNElEQVQ1f4dmjPGjHGsKIpKRNK7DSQy/iMhPIrJKRCxBBKg5c7YQFfUuPXpMI6PiV6tWBUsIxhiPNYVlQBOgSwHEYnzs0KGT/POfc/nggzUAVKlSikOHTlG5cik/R2aMKSw8JQUBUNWtF7JzEWkPvAWEAhNUdXiWz/8J9AJSgQPAA6q680KOZc5PVfn8c2cAu/37TxAeXoznn7+Rxx67lmLFLvSuZGNMMPKUFKq4LtzZUtU3zveZazTV0UBbIAlYLiKJqroxU7FVQKyqnhSR/sAI4E6vozceqSo9ekxjypT1ALRqVYP33utM3bqV/ByZMaYw8pQUQoHSuGoMudQM2KKq2wBE5BPgNsCdFFR1fqbyS4G7L+A4JgciQv36VShTpjgjRrSlT5+rbQA7Y8x5eUoKe1X1+Qvc9+X8dXykJOCaHMrHA7Oz+0BE+gB9AKpXt+kdPdm+PZlt25Jp08a5rXTQoJb07NmIiIiyfo7MGFPYeWpQzstPyuy2zfYZBxG5G4gFXs3uc1Udr6qxqhpbpUqVPIQU3NLS0nnrraVERb3LnXdOZf/+EwCEhYVaQjDGeMVTTaFNHvadBGS+xzEC+C1rIRG5GXgGuEFVT+fheEXaxo0H6NUrkSVLkgCIi7vSmomMMbmWY1JQ1cN52PdyIFJEauEMkdENuCtzARFpDIwD2rsm8jG5dPZsGq+8sph//3shZ86kUbVqGd59tyNxcVf6OzRjTADy2UhnqpoqIgOBOTgd1hNVdYOIPA+sUNVEnOai0sBnrvH5d6lqnK9iCkZ33TWNqVOdvvvevZvw6qttKVcu3M9RGWMClU+Hv1TVWcCsLOv+L9P7m315/KLgkUeuYfXqfYwb14nWrWv5OxxjTICzJ5cCzPff72DYsAXu5euuq86mTQ9aQjDG5AsbKD9A/PHHaQYN+oaxY1cCcNNNtWjVypnnyJ5KNsbkl6JzNenYEUScV4CZNWszDRqMYezYlYSFhTB06A00bx7h77CMMUGo6NQUZs3663KHDv6JIxcOHjzJo49+zeTJ6wBo1uxyEhLiiIq62M+RGWOCVdFJChkCaI6g55//nsmT11GyZDFeeKE1jzxyDaGhRadyZ4wpeEUvKRRyqorr9lyGDbuR338/wUsvtaZ27Yp+jswYUxRYUigkVJUJE35i4sTVzJ9/H+HhxahQoSSfftrV36EZ4zdnz54lKSmJlJQUf4cSMMLDw4mIiCAsLOyCtrekUAhs3XqY3r1nMH/+DgD++98N3HtvjH+DMqYQSEpKokyZMtSsWdNdgzbnp6ocOnSIpKQkatW6sNvUrYHaj9LS0nnjjSU0bPgu8+fvoEqVUnzyye3cc0+0v0MzplBISUmhUqVKlhC8JCJUqlQpTzUrqyn4yYYN+3nggUSWLdsDQI8eDRk5sr1NjWlMFpYQciev35clBT9ZtWofy5bt4fLLyzBuXCc6dqzr75CMMcaajwrSgQMn3O979GjI22/fyoYNAywhGFOI7dixg6ioKJ/se8GCBXTq1AmAxMREhg8f7mEL37OkUABOnjzLE0/MpWbNt9i06QDgVPEGDmxmI5oaYwCIi4tj8ODB/g7DkoKvzZ+/nejod3n99SWkpKSycOFOf4dkTMASGXbe1/jxK93lxo9fmWPZ3EpNTeW+++4jOjqarl27cvLkSZ5//nmaNm1KVFQUffr0QV0Pxo4aNYr69esTHR1Nt27dADhx4gQPPPAATZs2pXHjxkyfPv2cY0yaNImBAwcC0LNnTx5++GFatGjBFVdcwdSpU93lXn31VZo2bUp0dDRDhgzJ9bl4YknBR44eTaFv3xm0bv0BW7cm07Dhxfz4Yy/69o31d2jGmFz65Zdf6NOnD2vXrqVs2bKMGTOGgQMHsnz5ctavX8+pU6eYOXMmAMOHD2fVqlWsXbuWsWPHAvDiiy/SunVrli9fzvz583nyySc5ceJETodk7969LFq0iJkzZ7prEHPnzmXz5s0sW7aM1atXs3LlShYuXJiv52odzT6waNEuunWbyp49xwgLC+G551oxaNB1FC8e6u/QjAloqt79Mu7T52r69Lk6345brVo1WrZsCcDdd9/NqFGjqFWrFiNGjODkyZMcPnyYBg0a0LlzZ6Kjo+nRowddunShS5cugHMxT0xM5LXXXgOcW2137dqV4zG7dOlCSEgI9evX5/fff3fvZ+7cuTRu3BiA48ePs3nzZlq1apVv52pJwQcuvbQ0hw6donnzCCZM6EyDBjaAnTGBLOttniLCgAEDWLFiBdWqVWPo0KHuZwO++uorFi5cSGJiIv/+97/ZsGEDqsrnn3/OlVf+dZrcjIt9dkqUKOF+n9E0pao89dRT9O3bN79O7RzWfJQPVJW5c7e6/3B16lRk0aL7WbTofksIxgSBXbt2sWTJEgCmTJnCddddB0DlypU5fvy4u80/PT2d3bt3c9NNNzFixAiOHDnC8ePHueWWW3j77bfd14hVq1ZdUBy33HILEydO5Pjx4wDs2bOH/fvzd3p7qynk0e7dR+nf/yu++mozCQlxPPCAU627+uqqfo7MGJNfrrrqKt5//3369u1LZGQk/fv3Jzk5mYYNG1KzZk2aNm0KQFpaGnfffTdHjx5FVXnssccoX748zz33HI8++ijR0dGoKjVr1nT3QeRGu3bt2LRpE9deey0ApUuX5qOPPuLii/Pvx6doAA0lDRAbG6srVqzI/YYZ1b98Ot/0dOW991by5JPfcOzYGcqVK8Ho0R3o0cOGqDAmv2zatImrrrrK32EEnOy+NxFZqaoe73SxmsIF2Lz5EL17z+D7753bS7t0qcfo0R2oWrWMnyMzxpi8saSQS//7327atPmAlJRULr74It5551a6dq1v47MYY4KCdTTnUmxsVSIjK3LvvTFs3DiAf/yjgSWEAPbFF18gIvz888/udZmHHsjQs2dPd2fi2bNnGTx4MJGRkURFRdGsWTNmz56d43FOnz7NnXfeSZ06dbjmmmvYsWNHtuXeeustoqKiaNCgASNHjnSvX7NmDddeey0NGzakc+fO/PHHHxd4xsbkzJKCB6dPp/Liiws5ePAkAMWLh7J48QO8/34XKlWyEU0DXcadJJ988onX2zz33HPs3buX9evXs379embMmMGxY8dy3CYhIYEKFSqwZcsWHnvsMQYNGnROmfXr1/Pee++xbNky1qxZw8yZM9m8eTMAvXr1Yvjw4axbt46//e1vvPrqq7k7UWO8ZEkhB0uXJtGkyXiefXY+jz76tXt9mTIlctjKBIrjx4+zePFiEhISvE4KJ0+e5L333uPtt99230d+ySWXcMcdd+S43fTp07nvvvsA6Nq1K/PmzSPrTR6bNm2iefPmlCpVimLFinHDDTfwxRdfAM4TtRkPKLVt25bPP/88V+dqjLcsKWTjxIkzPPbY17RokcDGjQeoW7cSffvm39ORpnD48ssvad++PXXr1qVixYr89NNPHrfZsmUL1atXp2zZstl+3qtXL7K7O27Pnj1Uq1YNgGLFilGuXDkOHTr0lzJRUVEsXLiQQ4cOcfLkSWbNmsXu3bvdnyUmJgLw2Wefudcbk98sKWQxb942GjZ8l5EjfyQkRBg8uCVr1vTj+utr+Ds0k8+mTJniHrCsW7duTJkyBTj/JCXe9B1NmDCB2Nhz7/rL7tbvrPu76qqrGDRoEG3btqV9+/bExMRQrJhzL8jEiRMZPXo0V199NceOHaN48eIeYzGFT82aNTl48GCey/iS3X2Uya+/HqJt2w9RhUaNLiUhIY4mTS7zd1jGBw4dOsR3333H+vXrERHS0tIQEUaMGEGlSpVITk7+S/nDhw9TuXJl6tSpw65duzh27Bhlynh/C3JERAS7d+8mIiKC1NRUjh49SsWKFc8pFx8fT3x8PABPP/00ERERANSrV4+5c+cC8Ouvv/LVV19d6KkbkyOrKWRSt24lHnnkGl58sTXLlvWyhBDEpk6dyr333svOnTvZsWMHu3fvplatWixatIjIyEh+++03Nm3aBMDOnTtZs2YNjRo1olSpUsTHx/Pwww9z5swZwBnN8qOPPsrxeHFxcbz//vvuY7du3TrbmkfGkAW7du1i2rRpdO/e/S/r09PTeeGFF+jXr1/+fBGBRMQ3Lw927NhBvXr16NWrF1FRUfTo0YNvv/2Wli1bEhkZybJlyzh8+DBdunQhOjqa5s2bs3btWsD58dGuXTsaN25M3759/1Jj/Oijj2jWrBmNGjWib9++pKWl+eyryxVVDajX1VdfrRfEeZb5L6v27Tumd9zxmX733bYL26cJWDfccIPOnj37L+veeust7devn6qqLlq0SK+55hqNiYnR2NhYnTt3rrvc6dOn9cknn9TatWtrgwYNtFmzZvr111+rqmp8fLwuX778nOOdOnVKu3btqrVr19amTZvq1q1bVVV1z549euutt7rLXXfddXrVVVdpdHS0fvvtt+71I0eO1MjISI2MjNRBgwZpenp6/n0ZhdjGjRv/XMj4N5zfLw+2b9+uoaGhunbtWk1LS9MmTZro/fffr+np6frll1/qbbfdpgMHDtShQ4eqquq8efM0JiZGVVUfeughHTZsmKqqzpw5UwE9cOCAbty4UTt16qRnzpxRVdX+/fvr+++/r6qqNWrU0AMHDuTf9+b++lihXlxj/X6Rz+0rP5JCenq6fvDBaq1Y8RWFoRoT826R+UdmTCDJ7uJW0LZv36516tRxL99zzz360Ucfqarq1q1bNSYmRhs1auRO9KqqEREReuTIEY2JifnL+goVKuiBAwf07bff1ssuu0xjYmI0JiZG69atq0OGDFFV/ycFn/YpiEh74C0gFJigqsOzfF4C+AC4GjgE3KmqO3wZ065dR+nXbyazZ28BoF272owb18keQDPGnFfmYaxDQkLcyyEhIaSmprpvCMgs45qS3bVFVbnvvvt4+eWXfRTxhfNZn4KIhAKjgVuB+kB3EamfpVg8kKyqdYA3gVd8FU+GBg3GMHv2FipUCGfSpNv4+use1KxZ3teHNcYEsVatWjF58mTAeSK+cuXKlC1b9i/rZ8+e7b6BoU2bNkydOtXdV3T48GF27iwcU/X6sqbQDNiiqtsAROQT4DZgY6YytwFDXe+nAu+IiLiqOj5x/PgZbr/9Kt55pwOXXlraV4cxxhQhQ4cO5f777yc6OppSpUq5byoYMmQI3bt3p0mTJtxwww1Ur14dgPr16/PCCy/Qrl070tPTCQsLY/To0dSo4f9b3302dLaIdAXaq2ov1/I9wDWqOjBTmfWuMkmu5a2uMgez7KsP0AegevXqV19QRnVV4T6fuoHbb89aYTHGFEY2dPaFKaxDZ2fXSJ81A3lTBlUdD4wHZz6FC4rGlfxuv6CNjTGmaPDlcwpJQLVMyxHAb+crIyLFgHLAYR/GZIwxJge+TArLgUgRqSUixYFuQGKWMonAfa73XYHvfNmfYIwJPHZJyJ28fl8+SwqqmgoMBOYAm4D/quoGEXleROJcxRKASiKyBfgnMNhX8RhjAk94eDiHDh2yxOAlVeXQoUOEh4df8D6KzhzNxpiAc/bsWZKSkkhJSfF3KAEjPDyciIgIwsLC/rK+MHQ0G2NMnoSFhVGrVi1/h1Gk2IB4xhhj3CwpGGOMcbOkYIwxxi3gOppF5ABwoYOEVAb8N6WRf9g5Fw12zkVDXs65hqpW8VQo4JJCXojICm9634OJnXPRYOdcNBTEOVvzkTHGGDdLCsYYY9yKWlIY7+8A/MDOuWiwcy4afH7ORapPwRhjTM6KWk3BGGNMDiwpGGOMcQvKpCAi7UXkFxHZIiLnjLwqIiVE5FPX5z+KSM2CjzJ/eXHO/xSRjSKyVkTmiYj/5/3LI0/nnKlcVxFREQn42xe9OWcRucP1t94gIh8XdIz5zYv/t6uLyHwRWeX6/7uDP+LMLyIyUUT2u2amzO5zEZFRru9jrYg0ydcAVDWoXkAosBW4AigOrAHqZykzABjret8N+NTfcRfAOd8ElHK9718UztlVrgywEFgKxPo77gL4O0cCq4AKruWL/R13AZzzeKC/6319YIe/487jObcCmgDrz/N5B2A2zsyVzYEf8/P4wVhTaAZsUdVtqnoG+AS4LUuZ24D3Xe+nAm1EJLupQQOFx3NW1fmqetK1uBRnJrxA5s3fGeDfwAggGMZe9uacewOjVTUZQFX3F3CM+c2bc1agrOt9Oc6d4TGgqOpCcp6B8jbgA3UsBcqLyGX5dfxgTAqXA7szLSe51mVbRp3JgI4ClQokOt/w5pwzi8f5pRHIPJ6ziDQGqqnqzIIMzIe8+TvXBeqKyGIRWSoi7QssOt/w5pyHAneLSBIwC3ioYELzm9z+e8+VYJxPIbtf/Fnvu/WmTCDx+nxE5G4gFrjBpxH5Xo7nLCIhwJtAz4IKqAB483cuhtOEdCNObfAHEYlS1SM+js1XvDnn7sAkVX1dRK4FPnSdc7rvw/MLn16/grGmkARUy7QcwbnVSXcZESmGU+XMqbpW2HlzzojIzcAzQJyqni6g2HzF0zmXAaKABSKyA6ftNTHAO5u9/X97uqqeVdXtwC84SSJQeXPO8cB/AVR1CRCOM3BcsPLq3/uFCsaksByIFJFaIlIcpyM5MUuZROA+1/uuwHfq6sEJUB7P2dWUMg4nIQR6OzN4OGdVPaqqlVW1pqrWxOlHiVPVQJ7L1Zv/t7/EuakAEamM05y0rUCjzF/enPMuoA2AiFyFkxQOFGiUBSsRuNd1F1Jz4Kiq7s2vnQdd85GqporIQGAOzp0LE1V1g4g8D6xQ1UQgAaeKuQWnhtDNfxHnnZfn/CpQGvjM1ae+S1Xj/BZ0Hnl5zkHFy3OeA7QTkY1AGvCkqh7yX9R54+U5Pw68JyKP4TSj9AzkH3kiMgWn+a+yq59kCBAGoKpjcfpNOgBbgJPA/fl6/AD+7owxxuSzYGw+MsYYc4EsKRhjjHGzpGCMMcbNkoIxxhg3SwrGGGPcLCkYnxGRNBFZnelVM4eyNc83KmQuj7nANaLmGtdQD1dewD76ici9rvc9RaRqps8miEj9fI5zuYg08mKbR0Wk1AUca6SItMpy3Iy/SVfX+oy/1XoR+SzjOFnWzxCR8q71VUTk69zGYgo/SwrGl06paqNMrx0FdNweqhqDM+jhq7ndWFXHquoHrsWeQNVMn/VS1Y35EuWfcY7BuzgfBXKVFESkItDcNcha5uNm/E2mutZl/K2igDNAv2zWHwYeBFDVA8BeEWmZm3hM4WdJwRQoV43gBxH5yfVqkU2ZBiKyzPULda2IRLrW351p/TgRCfVwuIVAHde2bcQZb3+dOOPVl3CtHy5/zjPxmmvdUBF5wvUrOhaY7DpmSdcv7VgR6S8iIzLF3FNE3r7AOJeQaUAzEXlXRFaIMx/CMNe6h3GS03wRme9a105Elri+x89EpHQ2++4K5PYX/Q8Z31tOceI8Pd0jl/s2hZwlBeNLJTM1U3zhWrcfaKuqTYA7gVHZbNcPeEtVG+FclJNcwxfcCbR0rU/D8wWpM7BORMKBScCdqtoQ50n+/q5f0X8DGqhqNPBC5o1dv6JX8Ocv61OZPp4K/D3T8p3ApxcYZ3ucC2yGZ1Q1FogGbhCRaFUdhTO+zU2qepM4Q1g8C9zs+i5XAP/MZt8tgZVZ1k3O9Hf5y+jA4owFdiuwLsv6UJyhJDI/Kb4CuN7DuZkAE3TDXJhC5ZTrwphZGPCOqw09DWdsnqyWAM+ISAQwTVU3i0gb4GpguWuYjpI4CSY7k0XkFLADZxjlK4Htqvqr6/P3cZpB3sGZZ2GCiHwFeD3EtqoeEJFt4ow9s9l1jMWu/eYmzotwhm/IPHvWHSLSB+ff52U4E8eszbJtc9f6xa7jFMf53rK6jHPHAeqRzRhQJUVktev9DzhDwWReXxMnuXyTaZv9ZGpaM8HBkoIpaI8BvwMxODXVcya/UdWPReRHoCMwR0R64QwX/L6qPuXFMf5y0cv6azjTcVJFpBnOL+BuwECgdS7O5VPgDuBn4AtVVXGu0F7HiTOT2HBgNPB3EakFPAE0VdVkEZmEM8BbVgJ8o6rdPRzj1Hm2P6dcNgncvV5EyuEkzQf5s3YX7tq/CSLWfGQKWjlgr2us+3twfiX/hYhcAWxzNZkk4jSjzAO6isjFrjIVxft5pn8GaopIRjv5PcD3rjb4cqo6C6cTN7uL4jGcYbizMw3ogjOe/6eudbmKU1XP4jQDNXc1PZUFTgBHReQSnKac7GJZCrTMOCcRKSUi2dW6NpF9/0CuqOpR4GHgCREJc62uC+T5jjFTuFhSMAVtDHCfiCzFuaicyKbMncB6V7NFPZypBzfiXDznishanGYMr6YgVNUUnJEkPxORdUA6MBbnAjvTtb/vcWoxWU0CxmZ0NGfZbzKwEaihqstc63Idp6uv4nXgCVVdgzPH8gZgIk6TVIbxwGwRme+6+6cnMMV1nKU431VWX+GMuJlnqroKp2aTMarwTa79myBio6QaE+REZBHQKb9nXxORhcBtGfNBm+BgScGYICci1+D0DWTtrM7LPqvg3GH1pcfCJqBYUjDGGONmfQrGGGPcLCkYY4xxs6RgjDHGzZKCMcYYN0sKxhhj3P4fhAaWt/mmhyIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from ml_utils.classification import plot_roc\n",
    "\n",
    "plot_roc(y_test, lm.predict_proba(X_test)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1272d710>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVYAAAEWCAYAAAA997/vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHpNJREFUeJzt3XmcHHWdxvHPMxNCIISQg3DHIJcLCAjhNEAEOUVA7kNJADerEHa9VjlcARERQcVVlI3cRI5wSQxnPAIiAiYCgQAG5IwgIQdXDpJJvvtH1ZBOmOnp6VR3V808b171mu6q6qpvJ+SZX/2q6leKCMzMLDtNjS7AzKyrcbCamWXMwWpmljEHq5lZxhysZmYZc7CamWXMwWqZkLSapN9KelvSzSuxneMl3ZdlbY0g6W5JIxpdhzWGg7WbkXScpMmS3pP0ehoAwzLY9BHAOsCAiDiy2o1ExK8jYt8M6lmOpOGSQtJtK8zfNp0/qcLtnCNpbEfrRcQBEXFNleVawTlYuxFJXwMuAb5PEoKDgV8Ah2Sw+Y8A0yOiJYNt1cqbwG6SBpTMGwFMz2oHSvjfVXcXEZ66wQT0Bd4DjiyzzqokwftaOl0CrJouGw7MAL4OzAReB05Ml50LLAIWp/s4GTgHGFuy7SFAAD3S9yOBF4B3gReB40vmP1jyud2AvwJvpz93K1k2CTgP+HO6nfuAge18t9b6LwNOTec1p/O+A0wqWfenwKvAO8AUYPd0/v4rfM8nSuo4P61jAbBpOu+L6fJfAreUbP9C4PeAGv3/hafaTP7N2n3sCvQCbi+zzlnALsB2wLbATsC3S5avSxLQG5CE56WS+kXE2SSt4JsiYo2IuKJcIZJ6A/8LHBARfUjC8/E21usP3JmuOwD4MXDnCi3O44ATgUFAT+Ab5fYNXAuckL7eD5hG8kuk1F9J/gz6A9cDN0vqFRH3rPA9ty35zBeAUUAf4OUVtvd1YBtJIyXtTvJnNyLSlLWux8HafQwAZkX5Q/Xjge9GxMyIeJOkJfqFkuWL0+WLI+IuklbbFlXWsxTYWtJqEfF6RExrY53PAM9FxHUR0RIRNwDPAp8tWeeqiJgeEQuAcSSB2K6IeAjoL2kLkoC9to11xkbE7HSfPyJpyXf0Pa+OiGnpZxavsL35wOdJfjGMBU6LiBkdbM8KzMHafcwGBkrqUWad9Vm+tfVyOu+DbawQzPOBNTpbSETMA44GvgS8LulOSR+roJ7WmjYoef+vKuq5DhgNfIo2WvCSvi7pmfQKh7dIWukDO9jmq+UWRsSjJF0fIvkFYF2Yg7X7+AuwEDi0zDqvkZyEajWYDx8mV2oesHrJ+3VLF0bEvRGxD7AeSSv0VxXU01rTP6usqdV1wCnAXWlr8gPpofq3gKOAfhGxFkn/rlpLb2ebZQ/rJZ1K0vJ9Dfhm9aVbEThYu4mIeJvkJM2lkg6VtLqkVSQdIOmH6Wo3AN+WtLakgen6HV5a1I7HgT0kDZbUFzijdYGkdSQdnPa1vk/SpbCkjW3cBWyeXiLWQ9LRwJbAhCprAiAiXgT2JOlTXlEfoIXkCoIekr4DrFmy/A1gSGfO/EvaHPgeSXfAF4BvSirbZWHF5mDtRiLix8DXSE5IvUly+Doa+E26yveAycBU4Engb+m8avY1Ebgp3dYUlg/DJpITOq8Bc0hC7pQ2tjEbOChddzZJS++giJhVTU0rbPvBiGirNX4vcDfJJVgvk7TySw/zW29+mC3pbx3tJ+16GQtcGBFPRMRzwJnAdZJWXZnvYPkln5g0M8uWW6xmZhlzsJqZZczBamaWMQermVnGyl0s3i0cevlkn70rkOtP2KHRJVgVVu8pdbxW+1b7xOiK/50ueOznK7WvLLjFamaWsW7fYjWzAijYSIwOVjPLv6bmRlfQKQ5WM8u/leuirTsHq5nln7sCzMwy5harmVnG3GI1M8uYW6xmZhnzVQFmZhlzV4CZWcYK1hVQrF8DZtY9qanyqaNNSVdKminpqZJ550j6p6TH0+nAkmVnSHpe0t8l7VdJuW6xmln+ZdsVcDXwcz786POfRMTFy+1W2hI4BtiK5KnBv5O0eUS09Yy2D7jFamb519xc+dSBiHiA5FlrlTgEuDEi3k8fQvk8sFNHH3Kwmln+SRVPkkZJmlwyjapwL6MlTU27Cvql8zZg+YdJzkjnleVgNbP860Qfa0SMiYihJdOYCvbwS2ATYDvgdeBHrXtuY90Ox4Z1H6uZ5V+NrwqIiDeW7Uq/Ytnj2mcAG5WsuiHJY9vLcovVzPIvw6sC2ty8tF7J288BrVcMjAeOkbSqpI2BzYBHO9qeW6xmln8Ztlgl3QAMBwZKmgGcDQyXtB3JYf5LwH8ARMQ0SeOAp4EW4NSOrggAB6uZFUGGt7RGxLFtzL6izPrnA+d3Zh8OVjPLP9/SamaWsYLd0upgNbP8c4vVzCxjDlYzs4x5PFYzs4y5j9XMLGPuCjAzy5hbrGZm2ZKD1cwsWw5WM7OMqcnBamaWKbdYzcwy5mA1M8uYg9XMLGvFylUHq5nln1usZmYZa2rynVdmZplyi9XMLGvFylUHq5nln1usZmYZc7CamWXMt7SamWXMLVYzs4w5WM3MMuZgNTPLmIPVzCxrxcpVB6uZ5Z9vaTUzy5i7AszMslasXHWwdhUHbTWIfbZYGwkmPvsmv502k2O2X599thjIOwtbABj7138yZcbbDa7U2nLgfnvRe/XeNDU309zczPU33droknLFLVaru8H9erHPFmvz33c8Q8vSpZy9/+ZMfjUJ0PFPvcEdT77R4AqtEmOuvJZ+/fo1uoxccrDWgKR+ETG30XXk1YZrrcb0N99j0ZKlAEx7/V12GeJ/oNZ1FC1Yi3KqbbKk6yXtpaL9CdfBK3MXsOW6feizajM9m5vYfqO+DOy9CgCf2XIQlxy2JaN3H0Lvns0NrtTaI4lT/uNkjjvqMG69+aZGl5M7alLFU4fbkq6UNFPSUyXzLpL0rKSpkm6XtFbJsjMkPS/p75L2q6TeogTr5sD1wGjgaUlnSlq/daGkn0h6vI3p9LY2JmmUpMmSJr/0wG11+gq1M+Othdz+xL8454DNOXv/zXhpznyWLIW7n5nJl8Y9yVdve5q5CxZz4s4bNbpUa8dV117PDeNu4+e//BU33Xg9Uyb/tdEl5YqkiqcKXA3sv8K8icDWEbENMB04I93vlsAxwFbpZ34hqcMWSiG6AiJiCTABmCBpbeAC4BVJu0XEoxHx1U5ubwwwBuDQyydH5gU3wO+mz+J302cB8PmhGzB73iLeXtDywfKJz77JWftu1qjyrAODBq0DQP8BA9hr708z7amp7DB0xwZXlR9ZHqhGxAOShqww776Stw8DR6SvDwFujIj3gRclPQ/sBPyl3D6K0mJFUl9Jo4DxJC3Yk4Gp6bJOtVi7or69kt+RA3v3ZJcha/HAP+bQb7VVPli+85B+vDJ3QaPKszIWzJ/PvHnvffD6Lw/9mU023bzBVeWL1Jlp2RFpOo3q5O5OAu5OX28AvFqybEY6r6xCtFgljQV2BW4GToiI50qXd7bF2hV969Ob0GfVHrQsDcY89ArzFi3h3/cczMYDViOAme8u4pcPvtzoMq0Ns2fP5mtfGQ3AkiVLOODAg/jksN0bXFW+dKbFWnpEWsV+zgJagF+3zmprFx1tpxDBCowDRkZES4drdlNnTvj7h+Zdcv+LDajEOmvDjTZi3K13NLqMXGuqw0DXkkYABwF7R0RreM4ASk9ObAi81tG2CtEVEBHjHapm3VdnugKq2772B74FHBwR80sWjQeOkbSqpI2BzYBHO9peUVqsZtaNZdlilXQDMBwYKGkGcDbJVQCrAhPTboeHI+JLETFN0jjgaZIuglPTk+llOVjNLPeyvHo9Io5tY/YVZdY/Hzi/M/twsJpZ7hXtviAHq5nlXsFy1cFqZvnnga7NzDLmFquZWcbcx2pmlrGC5aqD1czyr0u2WCXtBgwpXT8irq1RTWZmyylYrnYcrJKuAzYBHgda7zgIwMFqZnVRj7ECslRJi3UosGXJoARmZnVVtK6ASi4OewpYt9aFmJm1p9aDsGStkhbrQJLHoTwKvN86MyIOrllVZmYlitZirSRYz6l1EWZm5RQsVzsO1oi4X9I6QOsDeB6NiJm1LcvMbJminbzqsI9V0lEkA7seCRwFPCLpiPKfMjPLTsZPaa25SroCzgJ2bG2lpk9J/R1wSy0LMzNrlZfArFQlwdq0wqH/bArySBcz6xoKlqsVBes9ku4FbkjfHw3cVbuSzMyW1+VarBHx35IOBz5J8ijYMRFxe80rMzNLFSxXKxsrICJuBW6tcS1mZm0q2lUB7QarpAcjYpikd0nGBvhgERARsWbNqzMzA5oK1mRtN1gjYlj6s0/9yjEz+7CC5WpF17FeV8k8M7Na6YrXsW5V+kZSD2CH2pRjZvZhBetiLdvHegZwJrCapHdaZwOLgDF1qM3MDCjeyat2uwIi4oK0f/WiiFgznfpExICIOKOONZpZN6dO/JcHldxB9aikvq1vJK0l6dAa1mRmtpwmVT7lQSXBenZEvN36JiLeAs6uXUlmZsvriiev2gpfP93VzOomJ3lZsUoCcrKkHwOXktwocBowpaZVmZmVKNoNApV0BZxGciXATcDNwELg1FoWZWZWqqlJFU95UMkgLPOA0+tQi5lZmwrWYC17HeslEfEVSb9l+bECAD9M0MzqJ+uuAEn/Bfw7ybX5v4qISyT1JzkyHwK8BBwVEXOr2X65FmvrbasXV7NhM7OsZBmrkrYmCdWdSLo575F0Zzrv9xHxA0mnkxypf6uafZQbhGVK+vP+ajZsZpaVjC+j+jfg4YiYn277fuBzwCHA8HSda4BJZB2skp6kjS6AVhGxTTU7NDPrrM6ck5I0ChhVMmtMRJTehv8UcL6kAcAC4EBgMrBORLwOEBGvSxpUbb3lugIOSn+2XgHQ2jVwPDC/2h2amXVWZ872pyHa7ngmEfGMpAuBicB7wBNAy8rWWKrcWAEvR8TLwCcj4psR8WQ6nQ7sl2URZmblZH3nVURcERHbR8QewBzgOeANSeul+1sPmFluG+VUch1rb0nDWt9I2g3oXe0Ozcw6K+uxAloP8yUNBg4jeVjqeGBEusoI4I5q663kzquTgSvTgVgCeBs4qdodmpl1Vg3GALg17WNdDJwaEXMl/QAYJ+lk4BXgyGo3XskNAlOAbSWtCah0QBYzs3rIOlYjYvc25s0G9s5i+5U8mmUdSVcAN0XE25K2TBPdzKwumptU8ZQHlfSxXg3cC6yfvp8OfKVWBZmZrahowwZWEqwDI2IcsBQgIlqAJTWtysyshFT5lAeVnLyal3byBoCkXUhOYJmZ1UXRhg2sJFi/RnIZwiaS/gysDRxR06rMzEoULFfLB6ukJqAXsCewBcnJub9HxOI61FYXN44c2ugSrBP67Ti60SVYFRY89vOV+nxe+k4rVTZYI2KppB9FxK7AtDrVZGa2nOaCBWslJ6/uk3S4ivYrw8y6jKI9pbXSPtbeQIukhSTdARERa9a0MjOzVF4Cs1KV3HnVpx6FmJm1p2gHzOXGYx0EnAlsCkwFfhAR79SrMDOzVkVrsZbrY70WmAf8DOgD/G9dKjIzW0FXukFg3Yg4K319r6S/1aMgM7MV9chLYlaoXLBKUj+WDSzTXPo+IubUujgzM8hPS7RS5YK1LzCF5Ufsam21BvDRWhVlZlaqy9zSGhFD6liHmVm7CparFV3HambWUEW7KsDBama5l5cBrCvlYDWz3CtYrpa9QaB/uQ/6qgAzqxdl/tSr2irXYp1Ccva/rW/kqwLMrG66TIs1IjauZyFmZu0pWrBW8pRWSfq8pP9J3w+WtFPtSzMzS3TFhwn+AtgVOC59/y5wac0qMjNbQXNT5VMeVHJVwM4Rsb2kxwAiYq6knjWuy8zsA13mzqsSiyU1s+wprWuTPgrbzKweulwfK8lwgbcDgySdDzwIfL+mVZmZlehKwwYCEBG/ljQF2Jvk0qtDI+KZmldmZpZq6irXsa5wg8BM4IbSZb5BwMzqJS8t0UpVeoPAYGBu+not4BXA17maWV30KFgna4c3CEi6DBgfEXel7w8APl2f8szMitdireTk1Y6toQoQEXcDe9auJDOz5TVJFU95UEmwzpL0bUlDJH1E0lnA7FoXZmbWKuurAiStJekWSc9KekbSrpL6S5oo6bn0Z79q660kWI8F1ia55Oo3wKB0nplZXTR1YqrQT4F7IuJjwLbAM8DpwO8jYjPg9+n7qlRyudUc4L8krQksjYj3qt2ZmVk1sjzET7NsD2AkQEQsAhZJOgQYnq52DTAJ+FY1+6hkEJaPp7ezPglMkzRF0tbV7MzMrBqd6WOVNErS5JJp1Aqb+yjwJnCVpMckXS6pN7BORLwOkP4cVG29ldzS+n/A1yLijwCShgNjgN2q3amZWWd0pr0aEWNIMqo9PYDtgdMi4hFJP2UlDvvbUkmXRO/WUAWIiElA7yyLMDMrJ+OTVzOAGRHxSPr+FpKgfUPSesn+tB7JjVFVqSRYX5D0P+lVAUMkfRt4sdodmpl1VpbjsUbEv4BXJW2RztobeBoYD4xI540A7qi23kq6Ak4CzgVuI2mRPwCcWO0Ozcw6qwbDrJ4G/DodAvUFkkxrAsZJOpnk7tIjq914JVcFzAX+s9odmJmtrKwv/I+Ix4GhbSzaO4vtlxuEZXy5D0bEwVkUYGbWkbw8cqVS5VqsuwKvkoxq9QidOzFnZpaZnDxxpWLlgnVdYB+Su6yOA+4EboiIafUozMysVdFarO3+IoiIJRFxT0SMAHYBngcmSTqtbtWZmZEcLlc65UHZk1eSVgU+Q9JqHULymJbbal+WmdkyzQVrsZY7eXUNsDVwN3BuRDxVt6rMzEoULFfLtli/AMwDNgf+s6SPQ0BExJo1rs3MDADl5iC/MuWeIFC0E3Fm1kV1pRarmVkudJmntJqZ5YVbrGZmGcvLs6wq5WA1s9wr2NOvHaxmln9d5qoAM7O8KFhPgIO1q3n//fc58YTjWbxoES1LlrDPvvtxymiP+pgHl519PAfssTVvznmXoUd+/4P5Xz5mT7509B60LFnKPX96irN+egf9+/bm+otOZoetPsLY8Q/z1QtvbmDljecWaxmSXgKGRsSsKj+/HbB+RNyVaWFdSM+ePbn8ymtYvXdvFi9ezMgvHMew3fdgm223a3Rp3d51v32Yy266n8vPO+GDeXsM3YyDhn+cHY+6gEWLW1i73xoALHx/Md/9xQS23HR9ttpkvUaVnBtF62MtzE0AknoA2wEHVvG5bkMSq/dOHknW0tJCS0tL8Y6juqg//+0fzHl7/nLzRh25OxdfNZFFi1sAeHNu8nT5+QsX8dDjL7Dw/cV1rzOPOvOU1jyoWeikj5MdB2wINAPnpYtOk/RZYBXgyIh4VlJ/4EqSx9LOB0ZFxFRJ5wDrkwwAMwsYBqwmaRhwATAB+Bnw8fS7nBMRd0gaSTJ4TC+SBx/uVavvmUdLlizh2CMP45VXXuHoY49jm222bXRJ1o5NPzKIT35iE8499bMsXLSYM358O1OefqXRZeVOPuKycrVsse4PvBYR20bE1sA96fxZEbE98EvgG+m8c4HHImIb4Ezg2pLt7AAcEhHHAd8BboqI7SLiJuAs4A8RsSPwKeCiNNAhGah7RER8KFRLnzt+xa/KPSW3mJqbmxl32x3c94f7eerJqTz33PRGl2Tt6NHcRL81V2ePEy7mzJ/8hrE/PKnRJeWSW6zLPAlcLOlCYEJE/CkdyKV12MEpwGHp62HA4QAR8QdJAyT1TZeNj4gF7exjX+BgSa0B3QsYnL6eGBFz2vpQ6XPHF7YQVX27AlhzzTXZcaedeejBP7HZZps3uhxrwz/feIvf/P4JACZPe5mlS4OB/dZgVtolYIl8xGXlatZijYjpJK3NJ4ELJH0nXfR++nMJy4K9rT+31sCbV2Y3Ag5PW7DbRcTgiHimgs91WXPmzOGdd94BYOHChTz8l4cYsvFHG1yVtee3k6YyfKfkl96mgwfRc5UeDtW2FGyk61r2sa4PzImIsZLeA0aWWf0B4HjgPEnDSboL3mnjcQzvAn1K3t9L0md7WkSEpE9ExGOZfYkCmvXmTL595uksXbqEpUuDfffbnz2Hf6rRZRlwzQUj2X2HzRi41ho8f895nHfZXVzzm7/wf+ccz+Sbz2TR4iV88TvXfbD+s3eeS5/evei5Sg8++6ltOOiUS3n2hX818Bs0Tl4O8SuliNocCUvaD7gIWAosBr4M3EJ6uZWkocDFETE8PXl1FbAxHz559V5EXJxusz9JmK5CcvJqPHAJsBvJ76qXIuKg9OTV0IgY3VGdXbkroCvqt2OHf6WWQwse+/lKJeNfX3i74n+nO360b8NTuGbBWhQO1mJxsBbTSgfri50I1o0bH6zd6hpPMysm33llZpaxgnWxOljNLP8KlqsOVjPLvzauEMo1B6uZ5V7BctXBamb5V7BcdbCaWQEULFkdrGaWe0W73Kow47GaWfclVT51vC31kvSopCckTZN0bjp/Y0mPSHpO0k2SelZbr4PVzHIvy2AlGQhqr4jYlmTw/P0l7QJcCPwkIjYD5gInV1uvg9XMck+d+K8jkWgdQmyVdAqSAfFvSedfAxxabb0OVjPLvYxbrEhqlvQ4MBOYCPwDeCsiWtJVZgAbVFuvg9XMcq8zw7GWPiEknUatuL2IWBIR25E8Omon4N/a2G3VAzT5qgAzy79OXBRQ+oSQCtZ9S9IkYBdgLUk90lbrhsBrnS804RarmeVels+8krS2pLXS16sBnwaeAf4IHJGuNgK4o9p63WI1s9zL+CrW9YBrJDWTNC7HRcQESU8DN0r6HvAYcEW1O3Cwmln+ZZisETEV+EQb818g6W9daQ5WM8u9ot155WA1s9zz6FZmZhkrWK46WM0s/zzQtZlZxgqWqw5WM8u/guWqg9XMCqBgyepgNbPc8+VWZmYZcx+rmVnGmhysZmZZK1ayOljNLPfcFWBmlrGC5aqD1czyzy1WM7OM+ZZWM7OMFStWHaxmVgAFa7A6WM0s/3znlZlZ1oqVqw5WM8u/guWqg9XM8q+Sx1rniYPVzHKvYLlKU6MLMDPratxiNbPcK1qL1cFqZrnny63MzDLmFquZWcYcrGZmGXNXgJlZxtxiNTPLWMFy1cFqZgVQsGR1sJpZ7hXtllZFRKNrsBqQNCoixjS6Dquc/866Dt/S2nWNanQB1mn+O+siHKxmZhlzsJqZZczB2nW5r654/HfWRfjklZlZxtxiNTPLmIPVzCxjDlazGpH0kqSBK/H57SQdmGVNVh8O1gKT1K/RNVhtSOoBbAd0KljTz1mD+S+h2CZLegS4HPhj+Exkw0jqDYwDNgSagfPSRadJ+iywCnBkRDwrqT9wJfBRYD4wKiKmSjoHWB8YAswChgGrSRoGXABMAH4GfJzk3+45EXGHpJHAZ4BeQG9gr5p/YSvLwVpsmwMHAKOBSyVdB1wdEa8BSPoJ8Kk2PndjRPygfmV2C/sDr0XEZwAk9QUuBGZFxPaSTgG+AXwROBd4LCIOlbQXcC1J6xRgB2BYRCxIA3NoRIxOt/l94A8RcZKktYBHJf0u/dyuwDYRMacu39bKcrAWWEQsIWnFTJC0Nkmr5hVJu0XEoxHx1cZW2K08CVws6UJgQkT8ScnAIbely6cAh6WvhwGHA0TEHyQNSIMYYHxELGhnH/sCB0v6Rvq+FzA4fT3RoZofDtaCS/9BHg2cCCwGTgampsvcYq2TiJguaQeSPtELJN2XLno//bmEZf/e2hqqqbUbZ16Z3Qg4PCL+vtxMaecOPmd15mAtMEljSQ4BbwZOiIjnSpe7xVo/ktYH5kTEWEnvASPLrP4AcDxwnqThJN0F7+jDQ+O9C/QpeX8vSZ/taRERkj4REY9l9iUsMw7WYhsHjIyIlkYXYnwcuEjSUpIjhy8Dt7Sz7jnAVZKmkpy8GtHOen8ETpf0OEk3z3nAJcBUJSn8EnBQVl/AsuNbWs3MMubrWM3MMuZgNTPLmIPVzCxjDlYzs4w5WM3MMuZgtYaR9DlJIeljHaw3Mr1OtNr9DJc0odrPm3WWg9Ua6VjgQeCYDtYbSTI4iVkhOFitISStAXyS5BbcY0rmf1PSk5KekPQDSUcAQ4FfS3pc0mql45xKGippUvp6J0kPSXos/blF/b+Zme+8ssY5FLgnvcd+jqTtgXXS+TtHxHxJ/SNijqTRwDciYjJAG7d+tnoW2CMiWiR9Gvg+6WAnZvXkYLVGOZbk9kyAG9P3TcBVETEfoIrRmvoC10jajGRQk1UyqtWsUxysVneSBpAMxry1pCAZGDqAW1k2ylM5LSzrxupVMv88kgG/PydpCDApo5LNOsV9rNYIRwDXRsRHImJIRGwEvAjMAU6StDpAOtI+fHiUp5dIBoSG5Q/1+wL/TF+PrE3pZh1zsFojHAvcvsK8W0nO/I8neeTM4yQj7gNcDVzWevKKZAT+n0r6E8k4p61+SDIW6p9JWsFmDeHRrczMMuYWq5lZxhysZmYZc7CamWXMwWpmljEHq5lZxhysZmYZc7CamWXs/wFbQJiDdw3TNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from ml_utils.classification import confusion_matrix_visual\n",
    "\n",
    "confusion_matrix_visual(y_test, preds, ['>=', 'shorter'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
